{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv2d_31\n",
      "LSUV initializing conv2d_31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\PythonWorkSpace\\MLND\\P5_Image_Classification\\lsuv_init.py:19: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"co..., outputs=Tensor(\"co...)`\n",
      "  intermediate_layer_model = Model(input=model.get_input_at(0), output=layer.get_output_at(0))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.32397e-07\n",
      "1.0\n",
      "activation_35\n",
      "conv2d_32\n",
      "LSUV initializing conv2d_32\n",
      "0.0612368\n",
      "1.0\n",
      "activation_36\n",
      "conv2d_33\n",
      "LSUV initializing conv2d_33\n",
      "0.0315809\n",
      "1.0\n",
      "activation_37\n",
      "conv2d_34\n",
      "LSUV initializing conv2d_34\n",
      "0.0261135\n",
      "1.0\n",
      "activation_38\n",
      "conv2d_35\n",
      "LSUV initializing conv2d_35\n",
      "0.0333353\n",
      "0.999999\n",
      "activation_39\n",
      "max_pooling2d_5\n",
      "dropout_9\n",
      "conv2d_36\n",
      "LSUV initializing conv2d_36\n",
      "0.0390778\n",
      "1.0\n",
      "activation_40\n",
      "conv2d_37\n",
      "LSUV initializing conv2d_37\n",
      "0.0208651\n",
      "1.0\n",
      "activation_41\n",
      "conv2d_38\n",
      "LSUV initializing conv2d_38\n",
      "0.0160643\n",
      "1.0\n",
      "activation_42\n",
      "conv2d_39\n",
      "LSUV initializing conv2d_39\n",
      "0.0214555\n",
      "1.0\n",
      "activation_43\n",
      "conv2d_40\n",
      "LSUV initializing conv2d_40\n",
      "0.0236732\n",
      "1.0\n",
      "activation_44\n",
      "max_pooling2d_6\n",
      "dropout_10\n",
      "conv2d_41\n",
      "LSUV initializing conv2d_41\n",
      "0.0170847\n",
      "1.0\n",
      "activation_45\n",
      "conv2d_42\n",
      "LSUV initializing conv2d_42\n",
      "0.0154044\n",
      "1.0\n",
      "activation_46\n",
      "conv2d_43\n",
      "LSUV initializing conv2d_43\n",
      "0.0105512\n",
      "1.0\n",
      "activation_47\n",
      "conv2d_44\n",
      "LSUV initializing conv2d_44\n",
      "0.0100923\n",
      "1.0\n",
      "activation_48\n",
      "conv2d_45\n",
      "LSUV initializing conv2d_45\n",
      "0.0101113\n",
      "1.0\n",
      "activation_49\n",
      "global_max_pooling2d_3\n",
      "dropout_11\n",
      "dense_5\n",
      "LSUV initializing dense_5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\PythonWorkSpace\\MLND\\P5_Image_Classification\\lsuv_init.py:19: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"co..., outputs=Tensor(\"de...)`\n",
      "  intermediate_layer_model = Model(input=model.get_input_at(0), output=layer.get_output_at(0))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.542027\n",
      "1.0\n",
      "activation_50\n",
      "dropout_12\n",
      "dense_6\n",
      "dense_6 too small\n",
      "activation_51\n",
      "LSUV: total layers initialized 16\n",
      "Using real-time data augmentation.\n",
      "Epoch 1/1600\n",
      "351/351 [==============================] - 13s - loss: 2.2214 - acc: 0.1455 - val_loss: 1.9645 - val_acc: 0.2402\n",
      "Epoch 2/1600\n",
      "351/351 [==============================] - 12s - loss: 1.8623 - acc: 0.2870 - val_loss: 1.6522 - val_acc: 0.3898\n",
      "Epoch 3/1600\n",
      "351/351 [==============================] - 12s - loss: 1.6850 - acc: 0.3633 - val_loss: 1.5094 - val_acc: 0.4410\n",
      "Epoch 4/1600\n",
      "351/351 [==============================] - 12s - loss: 1.5897 - acc: 0.4091 - val_loss: 1.4872 - val_acc: 0.4572\n",
      "Epoch 5/1600\n",
      "351/351 [==============================] - 12s - loss: 1.5187 - acc: 0.4374 - val_loss: 1.4429 - val_acc: 0.4798\n",
      "Epoch 6/1600\n",
      "351/351 [==============================] - 12s - loss: 1.4481 - acc: 0.4670 - val_loss: 1.2805 - val_acc: 0.5422\n",
      "Epoch 7/1600\n",
      "351/351 [==============================] - 12s - loss: 1.3904 - acc: 0.4919 - val_loss: 1.2993 - val_acc: 0.5238\n",
      "Epoch 8/1600\n",
      "351/351 [==============================] - 12s - loss: 1.3408 - acc: 0.5154 - val_loss: 1.2927 - val_acc: 0.5402\n",
      "Epoch 9/1600\n",
      "351/351 [==============================] - 12s - loss: 1.3062 - acc: 0.5267 - val_loss: 1.1957 - val_acc: 0.5664\n",
      "Epoch 10/1600\n",
      "351/351 [==============================] - 12s - loss: 1.2561 - acc: 0.5490 - val_loss: 1.0985 - val_acc: 0.6062\n",
      "Epoch 11/1600\n",
      "351/351 [==============================] - 12s - loss: 1.2289 - acc: 0.5600 - val_loss: 1.1264 - val_acc: 0.6000\n",
      "Epoch 12/1600\n",
      "351/351 [==============================] - 12s - loss: 1.1884 - acc: 0.5740 - val_loss: 1.0235 - val_acc: 0.6302\n",
      "Epoch 13/1600\n",
      "351/351 [==============================] - 12s - loss: 1.1567 - acc: 0.5861 - val_loss: 1.0517 - val_acc: 0.6248\n",
      "Epoch 14/1600\n",
      "351/351 [==============================] - 12s - loss: 1.1391 - acc: 0.5897 - val_loss: 0.9458 - val_acc: 0.6616\n",
      "Epoch 15/1600\n",
      "351/351 [==============================] - 12s - loss: 1.1058 - acc: 0.6060 - val_loss: 0.9994 - val_acc: 0.6462\n",
      "Epoch 16/1600\n",
      "351/351 [==============================] - 12s - loss: 1.0817 - acc: 0.6121 - val_loss: 0.9355 - val_acc: 0.6720\n",
      "Epoch 17/1600\n",
      "351/351 [==============================] - 12s - loss: 1.0634 - acc: 0.6217 - val_loss: 0.9392 - val_acc: 0.6650\n",
      "Epoch 18/1600\n",
      "351/351 [==============================] - 12s - loss: 1.0372 - acc: 0.6302 - val_loss: 0.9186 - val_acc: 0.6764\n",
      "Epoch 19/1600\n",
      "351/351 [==============================] - 12s - loss: 1.0372 - acc: 0.6305 - val_loss: 0.9072 - val_acc: 0.6756\n",
      "Epoch 20/1600\n",
      "351/351 [==============================] - 12s - loss: 1.0059 - acc: 0.6393 - val_loss: 0.8618 - val_acc: 0.6946\n",
      "Epoch 21/1600\n",
      "351/351 [==============================] - 12s - loss: 0.9900 - acc: 0.6491 - val_loss: 0.7967 - val_acc: 0.7140\n",
      "Epoch 22/1600\n",
      "351/351 [==============================] - 12s - loss: 0.9705 - acc: 0.6534 - val_loss: 0.8657 - val_acc: 0.6930\n",
      "Epoch 23/1600\n",
      "351/351 [==============================] - 12s - loss: 0.9530 - acc: 0.6620 - val_loss: 0.7835 - val_acc: 0.7210\n",
      "Epoch 24/1600\n",
      "351/351 [==============================] - 12s - loss: 0.9414 - acc: 0.6689 - val_loss: 0.7808 - val_acc: 0.7218\n",
      "Epoch 25/1600\n",
      "351/351 [==============================] - 12s - loss: 0.9227 - acc: 0.6747 - val_loss: 0.7756 - val_acc: 0.7312\n",
      "Epoch 26/1600\n",
      "351/351 [==============================] - 12s - loss: 0.9131 - acc: 0.6791 - val_loss: 0.8122 - val_acc: 0.7160\n",
      "Epoch 27/1600\n",
      "351/351 [==============================] - 12s - loss: 0.9008 - acc: 0.6797 - val_loss: 0.7330 - val_acc: 0.7424\n",
      "Epoch 28/1600\n",
      "351/351 [==============================] - 12s - loss: 0.8837 - acc: 0.6884 - val_loss: 0.7812 - val_acc: 0.7246\n",
      "Epoch 29/1600\n",
      "351/351 [==============================] - 12s - loss: 0.8698 - acc: 0.6941 - val_loss: 0.7730 - val_acc: 0.7276\n",
      "Epoch 30/1600\n",
      "351/351 [==============================] - 12s - loss: 0.8554 - acc: 0.6988 - val_loss: 0.7182 - val_acc: 0.7460\n",
      "Epoch 31/1600\n",
      "351/351 [==============================] - 12s - loss: 0.8397 - acc: 0.7049 - val_loss: 0.7202 - val_acc: 0.7476\n",
      "Epoch 32/1600\n",
      "351/351 [==============================] - 12s - loss: 0.8354 - acc: 0.7064 - val_loss: 0.7299 - val_acc: 0.7430\n",
      "Epoch 33/1600\n",
      "351/351 [==============================] - 12s - loss: 0.8237 - acc: 0.7118 - val_loss: 0.7426 - val_acc: 0.7426\n",
      "Epoch 34/1600\n",
      "351/351 [==============================] - 12s - loss: 0.8086 - acc: 0.7175 - val_loss: 0.6952 - val_acc: 0.7574\n",
      "Epoch 35/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7955 - acc: 0.7197 - val_loss: 0.6743 - val_acc: 0.7694\n",
      "Epoch 36/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7895 - acc: 0.7234 - val_loss: 0.6522 - val_acc: 0.7736\n",
      "Epoch 37/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7852 - acc: 0.7277 - val_loss: 0.7015 - val_acc: 0.7548\n",
      "Epoch 38/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7741 - acc: 0.7286 - val_loss: 0.6991 - val_acc: 0.7524\n",
      "Epoch 39/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7482 - acc: 0.7365 - val_loss: 0.6372 - val_acc: 0.7808\n",
      "Epoch 40/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7470 - acc: 0.7385 - val_loss: 0.6627 - val_acc: 0.7688\n",
      "Epoch 41/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7387 - acc: 0.7440 - val_loss: 0.6277 - val_acc: 0.7808\n",
      "Epoch 42/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7233 - acc: 0.7487 - val_loss: 0.6008 - val_acc: 0.7888\n",
      "Epoch 43/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7314 - acc: 0.7436 - val_loss: 0.5986 - val_acc: 0.7912\n",
      "Epoch 44/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7107 - acc: 0.7515 - val_loss: 0.6243 - val_acc: 0.7820\n",
      "Epoch 45/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7043 - acc: 0.7546 - val_loss: 0.6177 - val_acc: 0.7898\n",
      "Epoch 46/1600\n",
      "351/351 [==============================] - 12s - loss: 0.7010 - acc: 0.7569 - val_loss: 0.6039 - val_acc: 0.7922\n",
      "Epoch 47/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6891 - acc: 0.7591 - val_loss: 0.5968 - val_acc: 0.7954\n",
      "Epoch 48/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6791 - acc: 0.7641 - val_loss: 0.6067 - val_acc: 0.7938\n",
      "Epoch 49/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6717 - acc: 0.7656 - val_loss: 0.5670 - val_acc: 0.8086\n",
      "Epoch 50/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6640 - acc: 0.7697 - val_loss: 0.6072 - val_acc: 0.7882\n",
      "Epoch 51/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6639 - acc: 0.7683 - val_loss: 0.5543 - val_acc: 0.8100\n",
      "Epoch 52/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6494 - acc: 0.7726 - val_loss: 0.5698 - val_acc: 0.8012\n",
      "Epoch 53/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6434 - acc: 0.7774 - val_loss: 0.5297 - val_acc: 0.8192\n",
      "Epoch 54/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6420 - acc: 0.7760 - val_loss: 0.5477 - val_acc: 0.8070\n",
      "Epoch 55/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6211 - acc: 0.7830 - val_loss: 0.5502 - val_acc: 0.8126\n",
      "Epoch 56/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6346 - acc: 0.7794 - val_loss: 0.5234 - val_acc: 0.8168\n",
      "Epoch 57/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6209 - acc: 0.7842 - val_loss: 0.5288 - val_acc: 0.8178\n",
      "Epoch 58/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6098 - acc: 0.7884 - val_loss: 0.5095 - val_acc: 0.8248\n",
      "Epoch 59/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6068 - acc: 0.7880 - val_loss: 0.5260 - val_acc: 0.8182\n",
      "Epoch 60/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6032 - acc: 0.7925 - val_loss: 0.5288 - val_acc: 0.8100\n",
      "Epoch 61/1600\n",
      "351/351 [==============================] - 12s - loss: 0.6001 - acc: 0.7924 - val_loss: 0.5212 - val_acc: 0.8250\n",
      "Epoch 62/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5870 - acc: 0.7957 - val_loss: 0.5085 - val_acc: 0.8256\n",
      "Epoch 63/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5910 - acc: 0.7938 - val_loss: 0.4991 - val_acc: 0.8286\n",
      "Epoch 64/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5798 - acc: 0.7984 - val_loss: 0.4852 - val_acc: 0.8334\n",
      "Epoch 65/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5787 - acc: 0.8006 - val_loss: 0.4962 - val_acc: 0.8274\n",
      "Epoch 66/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5600 - acc: 0.8059 - val_loss: 0.4799 - val_acc: 0.8334\n",
      "Epoch 67/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5670 - acc: 0.8039 - val_loss: 0.4754 - val_acc: 0.8376\n",
      "Epoch 68/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5547 - acc: 0.8061 - val_loss: 0.4773 - val_acc: 0.8380\n",
      "Epoch 69/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5523 - acc: 0.8082 - val_loss: 0.4701 - val_acc: 0.8386\n",
      "Epoch 70/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5427 - acc: 0.8129 - val_loss: 0.4862 - val_acc: 0.8354\n",
      "Epoch 71/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5409 - acc: 0.8128 - val_loss: 0.4559 - val_acc: 0.8460\n",
      "Epoch 72/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5349 - acc: 0.8123 - val_loss: 0.4632 - val_acc: 0.8420\n",
      "Epoch 73/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5391 - acc: 0.8105 - val_loss: 0.4645 - val_acc: 0.8400\n",
      "Epoch 74/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5301 - acc: 0.8169 - val_loss: 0.4802 - val_acc: 0.8388\n",
      "Epoch 75/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5272 - acc: 0.8186 - val_loss: 0.4445 - val_acc: 0.8492\n",
      "Epoch 76/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5234 - acc: 0.8188 - val_loss: 0.4854 - val_acc: 0.8362\n",
      "Epoch 77/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5152 - acc: 0.8213 - val_loss: 0.4516 - val_acc: 0.8490\n",
      "Epoch 78/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5167 - acc: 0.8213 - val_loss: 0.4714 - val_acc: 0.8406\n",
      "Epoch 79/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5004 - acc: 0.8262 - val_loss: 0.4450 - val_acc: 0.8450\n",
      "Epoch 80/1600\n",
      "351/351 [==============================] - 12s - loss: 0.5102 - acc: 0.8218 - val_loss: 0.4690 - val_acc: 0.8416\n",
      "Epoch 81/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4967 - acc: 0.8280 - val_loss: 0.4496 - val_acc: 0.8440\n",
      "Epoch 82/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4955 - acc: 0.8275 - val_loss: 0.4667 - val_acc: 0.8424\n",
      "Epoch 83/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4932 - acc: 0.8313 - val_loss: 0.4345 - val_acc: 0.8580\n",
      "Epoch 84/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4927 - acc: 0.8271 - val_loss: 0.4663 - val_acc: 0.8460\n",
      "Epoch 85/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4890 - acc: 0.8304 - val_loss: 0.4941 - val_acc: 0.8322\n",
      "Epoch 86/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4827 - acc: 0.8320 - val_loss: 0.4362 - val_acc: 0.8552\n",
      "Epoch 87/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4797 - acc: 0.8330 - val_loss: 0.4324 - val_acc: 0.8554\n",
      "Epoch 88/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4788 - acc: 0.8335 - val_loss: 0.4371 - val_acc: 0.8536\n",
      "Epoch 89/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4716 - acc: 0.8360 - val_loss: 0.4489 - val_acc: 0.8462\n",
      "Epoch 90/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4654 - acc: 0.8371 - val_loss: 0.4327 - val_acc: 0.8574\n",
      "Epoch 91/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4595 - acc: 0.8377 - val_loss: 0.4376 - val_acc: 0.8528\n",
      "Epoch 92/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4577 - acc: 0.8401 - val_loss: 0.4390 - val_acc: 0.8512\n",
      "Epoch 93/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4585 - acc: 0.8400 - val_loss: 0.4409 - val_acc: 0.8578\n",
      "Epoch 94/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4462 - acc: 0.8429 - val_loss: 0.4180 - val_acc: 0.8620\n",
      "Epoch 95/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4516 - acc: 0.8435 - val_loss: 0.4229 - val_acc: 0.8568\n",
      "Epoch 96/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4458 - acc: 0.8449 - val_loss: 0.4278 - val_acc: 0.8552\n",
      "Epoch 97/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4421 - acc: 0.8456 - val_loss: 0.4127 - val_acc: 0.8620\n",
      "Epoch 98/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4420 - acc: 0.8447 - val_loss: 0.4369 - val_acc: 0.8498\n",
      "Epoch 99/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4389 - acc: 0.8475 - val_loss: 0.4122 - val_acc: 0.8620\n",
      "Epoch 100/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4376 - acc: 0.8470 - val_loss: 0.4175 - val_acc: 0.8604\n",
      "Epoch 101/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4365 - acc: 0.8483 - val_loss: 0.4104 - val_acc: 0.8596\n",
      "Epoch 102/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4379 - acc: 0.8478 - val_loss: 0.4311 - val_acc: 0.8550\n",
      "Epoch 103/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4245 - acc: 0.8531 - val_loss: 0.4026 - val_acc: 0.8618\n",
      "Epoch 104/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4266 - acc: 0.8517 - val_loss: 0.4187 - val_acc: 0.8584\n",
      "Epoch 105/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4179 - acc: 0.8536 - val_loss: 0.4036 - val_acc: 0.8610\n",
      "Epoch 106/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4199 - acc: 0.8525 - val_loss: 0.4002 - val_acc: 0.8640\n",
      "Epoch 107/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4204 - acc: 0.8530 - val_loss: 0.4019 - val_acc: 0.8670\n",
      "Epoch 108/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4064 - acc: 0.8580 - val_loss: 0.4231 - val_acc: 0.8580\n",
      "Epoch 109/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4132 - acc: 0.8546 - val_loss: 0.3878 - val_acc: 0.8690\n",
      "Epoch 110/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4135 - acc: 0.8567 - val_loss: 0.3870 - val_acc: 0.8706\n",
      "Epoch 111/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4028 - acc: 0.8597 - val_loss: 0.3991 - val_acc: 0.8684\n",
      "Epoch 112/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4051 - acc: 0.8583 - val_loss: 0.4026 - val_acc: 0.8644\n",
      "Epoch 113/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4035 - acc: 0.8594 - val_loss: 0.4170 - val_acc: 0.8594\n",
      "Epoch 114/1600\n",
      "351/351 [==============================] - 12s - loss: 0.4003 - acc: 0.8600 - val_loss: 0.3956 - val_acc: 0.8648\n",
      "Epoch 115/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3989 - acc: 0.8599 - val_loss: 0.3901 - val_acc: 0.8702\n",
      "Epoch 116/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3961 - acc: 0.8616 - val_loss: 0.3782 - val_acc: 0.8750\n",
      "Epoch 117/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3952 - acc: 0.8617 - val_loss: 0.4226 - val_acc: 0.8572\n",
      "Epoch 118/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3880 - acc: 0.8620 - val_loss: 0.3925 - val_acc: 0.8724\n",
      "Epoch 119/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3828 - acc: 0.8662 - val_loss: 0.3891 - val_acc: 0.8688\n",
      "Epoch 120/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3890 - acc: 0.8620 - val_loss: 0.3863 - val_acc: 0.8700\n",
      "Epoch 121/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3779 - acc: 0.8686 - val_loss: 0.3879 - val_acc: 0.8738\n",
      "Epoch 122/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3747 - acc: 0.8684 - val_loss: 0.3747 - val_acc: 0.8688\n",
      "Epoch 123/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3785 - acc: 0.8693 - val_loss: 0.3843 - val_acc: 0.8720\n",
      "Epoch 124/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3748 - acc: 0.8694 - val_loss: 0.3823 - val_acc: 0.8740\n",
      "Epoch 125/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3727 - acc: 0.8702 - val_loss: 0.3925 - val_acc: 0.8670\n",
      "Epoch 126/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3720 - acc: 0.8699 - val_loss: 0.3827 - val_acc: 0.8688\n",
      "Epoch 127/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3748 - acc: 0.8698 - val_loss: 0.3744 - val_acc: 0.8780\n",
      "Epoch 128/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3639 - acc: 0.8726 - val_loss: 0.3799 - val_acc: 0.8698\n",
      "Epoch 129/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3687 - acc: 0.8707 - val_loss: 0.3847 - val_acc: 0.8696\n",
      "Epoch 130/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3589 - acc: 0.8757 - val_loss: 0.3693 - val_acc: 0.8766\n",
      "Epoch 131/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3547 - acc: 0.8766 - val_loss: 0.3816 - val_acc: 0.8740\n",
      "Epoch 132/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3522 - acc: 0.8758 - val_loss: 0.3755 - val_acc: 0.8726\n",
      "Epoch 133/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3556 - acc: 0.8765 - val_loss: 0.3680 - val_acc: 0.8756\n",
      "Epoch 134/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3619 - acc: 0.8734 - val_loss: 0.3676 - val_acc: 0.8750\n",
      "Epoch 135/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3409 - acc: 0.8816 - val_loss: 0.3885 - val_acc: 0.8776\n",
      "Epoch 136/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3470 - acc: 0.8795 - val_loss: 0.3709 - val_acc: 0.8758\n",
      "Epoch 137/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3497 - acc: 0.8786 - val_loss: 0.3873 - val_acc: 0.8726\n",
      "Epoch 138/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3427 - acc: 0.8802 - val_loss: 0.3698 - val_acc: 0.8802\n",
      "Epoch 139/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3383 - acc: 0.8816 - val_loss: 0.3824 - val_acc: 0.8748\n",
      "Epoch 140/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3440 - acc: 0.8790 - val_loss: 0.3617 - val_acc: 0.8796\n",
      "Epoch 141/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3382 - acc: 0.8833 - val_loss: 0.3741 - val_acc: 0.8772\n",
      "Epoch 142/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3370 - acc: 0.8828 - val_loss: 0.3608 - val_acc: 0.8760\n",
      "Epoch 143/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3365 - acc: 0.8830 - val_loss: 0.3676 - val_acc: 0.8742\n",
      "Epoch 144/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3298 - acc: 0.8840 - val_loss: 0.3718 - val_acc: 0.8796\n",
      "Epoch 145/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3303 - acc: 0.8856 - val_loss: 0.3839 - val_acc: 0.8742\n",
      "Epoch 146/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3308 - acc: 0.8828 - val_loss: 0.3881 - val_acc: 0.8744\n",
      "Epoch 147/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3290 - acc: 0.8854 - val_loss: 0.3667 - val_acc: 0.8732\n",
      "Epoch 148/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3187 - acc: 0.8886 - val_loss: 0.3714 - val_acc: 0.8760\n",
      "Epoch 149/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3327 - acc: 0.8844 - val_loss: 0.3617 - val_acc: 0.8776\n",
      "Epoch 150/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3218 - acc: 0.8865 - val_loss: 0.3709 - val_acc: 0.8770\n",
      "Epoch 151/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3176 - acc: 0.8890 - val_loss: 0.3833 - val_acc: 0.8696\n",
      "Epoch 152/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3185 - acc: 0.8884 - val_loss: 0.3654 - val_acc: 0.8764\n",
      "Epoch 153/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3116 - acc: 0.8910 - val_loss: 0.3724 - val_acc: 0.8760\n",
      "Epoch 154/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3179 - acc: 0.8896 - val_loss: 0.3646 - val_acc: 0.8786\n",
      "Epoch 155/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3174 - acc: 0.8886 - val_loss: 0.3708 - val_acc: 0.8756\n",
      "Epoch 156/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3076 - acc: 0.8927 - val_loss: 0.3738 - val_acc: 0.8794\n",
      "Epoch 157/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3163 - acc: 0.8891 - val_loss: 0.3669 - val_acc: 0.8756\n",
      "Epoch 158/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3139 - acc: 0.8900 - val_loss: 0.3717 - val_acc: 0.8786\n",
      "Epoch 159/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3097 - acc: 0.8913 - val_loss: 0.3664 - val_acc: 0.8782\n",
      "Epoch 160/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3052 - acc: 0.8926 - val_loss: 0.3674 - val_acc: 0.8774\n",
      "Epoch 161/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3015 - acc: 0.8957 - val_loss: 0.3813 - val_acc: 0.8754\n",
      "Epoch 162/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3041 - acc: 0.8934 - val_loss: 0.3544 - val_acc: 0.8846\n",
      "Epoch 163/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3028 - acc: 0.8931 - val_loss: 0.3629 - val_acc: 0.8828\n",
      "Epoch 164/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2981 - acc: 0.8961 - val_loss: 0.3483 - val_acc: 0.8838\n",
      "Epoch 165/1600\n",
      "351/351 [==============================] - 12s - loss: 0.3034 - acc: 0.8932 - val_loss: 0.3595 - val_acc: 0.8826\n",
      "Epoch 166/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2947 - acc: 0.8952 - val_loss: 0.3931 - val_acc: 0.8772\n",
      "Epoch 167/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2977 - acc: 0.8962 - val_loss: 0.3757 - val_acc: 0.8784\n",
      "Epoch 168/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2911 - acc: 0.8960 - val_loss: 0.3704 - val_acc: 0.8788\n",
      "Epoch 169/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2981 - acc: 0.8969 - val_loss: 0.3649 - val_acc: 0.8778\n",
      "Epoch 170/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2913 - acc: 0.8977 - val_loss: 0.3505 - val_acc: 0.8798\n",
      "Epoch 171/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2856 - acc: 0.9005 - val_loss: 0.3710 - val_acc: 0.8804\n",
      "Epoch 172/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2926 - acc: 0.8974 - val_loss: 0.3645 - val_acc: 0.8826\n",
      "Epoch 173/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2845 - acc: 0.9003 - val_loss: 0.3616 - val_acc: 0.8820\n",
      "Epoch 174/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2850 - acc: 0.8994 - val_loss: 0.3526 - val_acc: 0.8854\n",
      "Epoch 175/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2821 - acc: 0.9001 - val_loss: 0.3753 - val_acc: 0.8762\n",
      "Epoch 176/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2907 - acc: 0.9000 - val_loss: 0.3522 - val_acc: 0.8844\n",
      "Epoch 177/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2811 - acc: 0.9008 - val_loss: 0.3684 - val_acc: 0.8806\n",
      "Epoch 178/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2775 - acc: 0.9028 - val_loss: 0.3848 - val_acc: 0.8788\n",
      "Epoch 179/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2779 - acc: 0.9023 - val_loss: 0.3608 - val_acc: 0.8860\n",
      "Epoch 180/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2779 - acc: 0.9021 - val_loss: 0.3630 - val_acc: 0.8842\n",
      "Epoch 181/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2731 - acc: 0.9037 - val_loss: 0.3614 - val_acc: 0.8840\n",
      "Epoch 182/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2785 - acc: 0.9030 - val_loss: 0.3482 - val_acc: 0.8848\n",
      "Epoch 183/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2758 - acc: 0.9029 - val_loss: 0.3611 - val_acc: 0.8868\n",
      "Epoch 184/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2657 - acc: 0.9076 - val_loss: 0.3535 - val_acc: 0.8880\n",
      "Epoch 185/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2709 - acc: 0.9059 - val_loss: 0.3855 - val_acc: 0.8804\n",
      "Epoch 186/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2669 - acc: 0.9061 - val_loss: 0.3834 - val_acc: 0.8742\n",
      "Epoch 187/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2682 - acc: 0.9062 - val_loss: 0.3605 - val_acc: 0.8852\n",
      "Epoch 188/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2690 - acc: 0.9055 - val_loss: 0.3710 - val_acc: 0.8822\n",
      "Epoch 189/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2680 - acc: 0.9064 - val_loss: 0.3713 - val_acc: 0.8846\n",
      "Epoch 190/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2675 - acc: 0.9076 - val_loss: 0.3815 - val_acc: 0.8776\n",
      "Epoch 191/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2601 - acc: 0.9095 - val_loss: 0.3625 - val_acc: 0.8870\n",
      "Epoch 192/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2561 - acc: 0.9106 - val_loss: 0.3560 - val_acc: 0.8872\n",
      "Epoch 193/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2620 - acc: 0.9084 - val_loss: 0.3577 - val_acc: 0.8866\n",
      "Epoch 194/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2560 - acc: 0.9100 - val_loss: 0.3682 - val_acc: 0.8828\n",
      "Epoch 195/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2583 - acc: 0.9093 - val_loss: 0.3676 - val_acc: 0.8864\n",
      "Epoch 196/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2586 - acc: 0.9089 - val_loss: 0.3589 - val_acc: 0.8846\n",
      "Epoch 197/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2545 - acc: 0.9102 - val_loss: 0.3546 - val_acc: 0.8852\n",
      "Epoch 198/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2545 - acc: 0.9091 - val_loss: 0.3519 - val_acc: 0.8840\n",
      "Epoch 199/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2562 - acc: 0.9099 - val_loss: 0.3505 - val_acc: 0.8880\n",
      "Epoch 200/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2478 - acc: 0.9124 - val_loss: 0.3589 - val_acc: 0.8854\n",
      "Epoch 201/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2531 - acc: 0.9108 - val_loss: 0.3746 - val_acc: 0.8828\n",
      "Epoch 202/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2556 - acc: 0.9090 - val_loss: 0.3664 - val_acc: 0.8822\n",
      "Epoch 203/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2459 - acc: 0.9134 - val_loss: 0.3624 - val_acc: 0.8818\n",
      "Epoch 204/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2445 - acc: 0.9135 - val_loss: 0.3646 - val_acc: 0.8872\n",
      "Epoch 205/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2459 - acc: 0.9144 - val_loss: 0.3625 - val_acc: 0.8808\n",
      "Epoch 206/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2444 - acc: 0.9138 - val_loss: 0.3804 - val_acc: 0.8764\n",
      "Epoch 207/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2460 - acc: 0.9128 - val_loss: 0.3473 - val_acc: 0.8862\n",
      "Epoch 208/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2428 - acc: 0.9150 - val_loss: 0.3702 - val_acc: 0.8828\n",
      "Epoch 209/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2396 - acc: 0.9154 - val_loss: 0.3528 - val_acc: 0.8838\n",
      "Epoch 210/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2455 - acc: 0.9150 - val_loss: 0.3615 - val_acc: 0.8874\n",
      "Epoch 211/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2389 - acc: 0.9152 - val_loss: 0.3796 - val_acc: 0.8854\n",
      "Epoch 212/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2414 - acc: 0.9149 - val_loss: 0.3486 - val_acc: 0.8876\n",
      "Epoch 213/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2436 - acc: 0.9140 - val_loss: 0.3485 - val_acc: 0.8876\n",
      "Epoch 214/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2333 - acc: 0.9191 - val_loss: 0.3581 - val_acc: 0.8816\n",
      "Epoch 215/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2335 - acc: 0.9188 - val_loss: 0.3857 - val_acc: 0.8766\n",
      "Epoch 216/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2325 - acc: 0.9176 - val_loss: 0.3474 - val_acc: 0.8906\n",
      "Epoch 217/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2373 - acc: 0.9177 - val_loss: 0.3562 - val_acc: 0.8840\n",
      "Epoch 218/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2342 - acc: 0.9166 - val_loss: 0.3725 - val_acc: 0.8880\n",
      "Epoch 219/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2381 - acc: 0.9152 - val_loss: 0.3552 - val_acc: 0.8860\n",
      "Epoch 220/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2334 - acc: 0.9178 - val_loss: 0.3608 - val_acc: 0.8846\n",
      "Epoch 221/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2305 - acc: 0.9182 - val_loss: 0.3455 - val_acc: 0.8918\n",
      "Epoch 222/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2322 - acc: 0.9178 - val_loss: 0.3601 - val_acc: 0.8870\n",
      "Epoch 223/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2248 - acc: 0.9214 - val_loss: 0.3452 - val_acc: 0.8912\n",
      "Epoch 224/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2297 - acc: 0.9208 - val_loss: 0.3679 - val_acc: 0.8868\n",
      "Epoch 225/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2253 - acc: 0.9214 - val_loss: 0.3583 - val_acc: 0.8854\n",
      "Epoch 226/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2234 - acc: 0.9210 - val_loss: 0.3677 - val_acc: 0.8864\n",
      "Epoch 227/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2219 - acc: 0.9217 - val_loss: 0.3497 - val_acc: 0.8924\n",
      "Epoch 228/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2213 - acc: 0.9228 - val_loss: 0.3717 - val_acc: 0.8884\n",
      "Epoch 229/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2228 - acc: 0.9214 - val_loss: 0.3773 - val_acc: 0.8810\n",
      "Epoch 230/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2225 - acc: 0.9219 - val_loss: 0.3657 - val_acc: 0.8862\n",
      "Epoch 231/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2205 - acc: 0.9212 - val_loss: 0.3645 - val_acc: 0.8856\n",
      "Epoch 232/1600\n",
      "351/351 [==============================] - 11s - loss: 0.2198 - acc: 0.9233 - val_loss: 0.3697 - val_acc: 0.8876\n",
      "Epoch 233/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2221 - acc: 0.9217 - val_loss: 0.3513 - val_acc: 0.8852\n",
      "Epoch 234/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2218 - acc: 0.9222 - val_loss: 0.3595 - val_acc: 0.8900\n",
      "Epoch 235/1600\n",
      "351/351 [==============================] - 11s - loss: 0.2161 - acc: 0.9244 - val_loss: 0.3635 - val_acc: 0.8910\n",
      "Epoch 236/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2168 - acc: 0.9230 - val_loss: 0.3715 - val_acc: 0.8874\n",
      "Epoch 237/1600\n",
      "351/351 [==============================] - 11s - loss: 0.2140 - acc: 0.9256 - val_loss: 0.3803 - val_acc: 0.8902\n",
      "Epoch 238/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2146 - acc: 0.9251 - val_loss: 0.3664 - val_acc: 0.8876\n",
      "Epoch 239/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2165 - acc: 0.9247 - val_loss: 0.3641 - val_acc: 0.8860\n",
      "Epoch 240/1600\n",
      "351/351 [==============================] - 11s - loss: 0.2100 - acc: 0.9260 - val_loss: 0.3693 - val_acc: 0.8908\n",
      "Epoch 241/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2138 - acc: 0.9250 - val_loss: 0.3844 - val_acc: 0.8860\n",
      "Epoch 242/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2120 - acc: 0.9234 - val_loss: 0.3513 - val_acc: 0.8902\n",
      "Epoch 243/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2112 - acc: 0.9252 - val_loss: 0.3903 - val_acc: 0.8832\n",
      "Epoch 244/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2081 - acc: 0.9273 - val_loss: 0.3811 - val_acc: 0.8908\n",
      "Epoch 245/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2146 - acc: 0.9253 - val_loss: 0.3780 - val_acc: 0.8810\n",
      "Epoch 246/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2101 - acc: 0.9259 - val_loss: 0.3593 - val_acc: 0.8890\n",
      "Epoch 247/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2098 - acc: 0.9264 - val_loss: 0.3824 - val_acc: 0.8838\n",
      "Epoch 248/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2078 - acc: 0.9279 - val_loss: 0.3654 - val_acc: 0.8870\n",
      "Epoch 249/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2129 - acc: 0.9242 - val_loss: 0.3860 - val_acc: 0.8868\n",
      "Epoch 250/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2045 - acc: 0.9285 - val_loss: 0.3616 - val_acc: 0.8890\n",
      "Epoch 251/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2035 - acc: 0.9281 - val_loss: 0.3726 - val_acc: 0.8868\n",
      "Epoch 252/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2004 - acc: 0.9291 - val_loss: 0.3694 - val_acc: 0.8916\n",
      "Epoch 253/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2090 - acc: 0.9269 - val_loss: 0.3693 - val_acc: 0.8896\n",
      "Epoch 254/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1971 - acc: 0.9305 - val_loss: 0.3695 - val_acc: 0.8884\n",
      "Epoch 255/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2001 - acc: 0.9293 - val_loss: 0.3696 - val_acc: 0.8884\n",
      "Epoch 256/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2059 - acc: 0.9271 - val_loss: 0.3983 - val_acc: 0.8794\n",
      "Epoch 257/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2048 - acc: 0.9271 - val_loss: 0.3712 - val_acc: 0.8826\n",
      "Epoch 258/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1996 - acc: 0.9307 - val_loss: 0.3800 - val_acc: 0.8834\n",
      "Epoch 259/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1968 - acc: 0.9298 - val_loss: 0.3806 - val_acc: 0.8868\n",
      "Epoch 260/1600\n",
      "351/351 [==============================] - 12s - loss: 0.2042 - acc: 0.9289 - val_loss: 0.3662 - val_acc: 0.8888\n",
      "Epoch 261/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1952 - acc: 0.9326 - val_loss: 0.3770 - val_acc: 0.8860\n",
      "Epoch 262/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1940 - acc: 0.9324 - val_loss: 0.3844 - val_acc: 0.8826\n",
      "Epoch 263/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1951 - acc: 0.9312 - val_loss: 0.3553 - val_acc: 0.8952\n",
      "Epoch 264/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1960 - acc: 0.9315 - val_loss: 0.3511 - val_acc: 0.8898\n",
      "Epoch 265/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1883 - acc: 0.9343 - val_loss: 0.3571 - val_acc: 0.8934\n",
      "Epoch 266/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1911 - acc: 0.9328 - val_loss: 0.3856 - val_acc: 0.8872\n",
      "Epoch 267/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1942 - acc: 0.9323 - val_loss: 0.3909 - val_acc: 0.8870\n",
      "Epoch 268/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1918 - acc: 0.9337 - val_loss: 0.3724 - val_acc: 0.8912\n",
      "Epoch 269/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1961 - acc: 0.9304 - val_loss: 0.3490 - val_acc: 0.8904\n",
      "Epoch 270/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1962 - acc: 0.9313 - val_loss: 0.3457 - val_acc: 0.8932\n",
      "Epoch 271/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1901 - acc: 0.9334 - val_loss: 0.3873 - val_acc: 0.8900\n",
      "Epoch 272/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1895 - acc: 0.9327 - val_loss: 0.3807 - val_acc: 0.8880\n",
      "Epoch 273/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1915 - acc: 0.9331 - val_loss: 0.3770 - val_acc: 0.8866\n",
      "Epoch 274/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1911 - acc: 0.9329 - val_loss: 0.3829 - val_acc: 0.8848\n",
      "Epoch 275/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1913 - acc: 0.9335 - val_loss: 0.3566 - val_acc: 0.8906\n",
      "Epoch 276/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1901 - acc: 0.9336 - val_loss: 0.3550 - val_acc: 0.8954\n",
      "Epoch 277/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1938 - acc: 0.9331 - val_loss: 0.3639 - val_acc: 0.8918\n",
      "Epoch 278/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1897 - acc: 0.9341 - val_loss: 0.3830 - val_acc: 0.8890\n",
      "Epoch 279/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1918 - acc: 0.9330 - val_loss: 0.3598 - val_acc: 0.8900\n",
      "Epoch 280/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1844 - acc: 0.9365 - val_loss: 0.3670 - val_acc: 0.8950\n",
      "Epoch 281/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1838 - acc: 0.9352 - val_loss: 0.3706 - val_acc: 0.8906\n",
      "Epoch 282/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1812 - acc: 0.9369 - val_loss: 0.3685 - val_acc: 0.8918\n",
      "Epoch 283/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1841 - acc: 0.9352 - val_loss: 0.3772 - val_acc: 0.8872\n",
      "Epoch 284/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1845 - acc: 0.9345 - val_loss: 0.3651 - val_acc: 0.8878\n",
      "Epoch 285/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1777 - acc: 0.9376 - val_loss: 0.3654 - val_acc: 0.8942\n",
      "Epoch 286/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1794 - acc: 0.9367 - val_loss: 0.3877 - val_acc: 0.8862\n",
      "Epoch 287/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1790 - acc: 0.9373 - val_loss: 0.3740 - val_acc: 0.8894\n",
      "Epoch 288/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1789 - acc: 0.9381 - val_loss: 0.3526 - val_acc: 0.8938\n",
      "Epoch 289/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1790 - acc: 0.9364 - val_loss: 0.3674 - val_acc: 0.8934\n",
      "Epoch 290/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1841 - acc: 0.9356 - val_loss: 0.3569 - val_acc: 0.8906\n",
      "Epoch 291/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1808 - acc: 0.9385 - val_loss: 0.3705 - val_acc: 0.8908\n",
      "Epoch 292/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1773 - acc: 0.9374 - val_loss: 0.3876 - val_acc: 0.8876\n",
      "Epoch 293/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1766 - acc: 0.9383 - val_loss: 0.3626 - val_acc: 0.8960\n",
      "Epoch 294/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1767 - acc: 0.9385 - val_loss: 0.3705 - val_acc: 0.8888\n",
      "Epoch 295/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1776 - acc: 0.9368 - val_loss: 0.3828 - val_acc: 0.8854\n",
      "Epoch 296/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1748 - acc: 0.9387 - val_loss: 0.3818 - val_acc: 0.8882\n",
      "Epoch 297/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1754 - acc: 0.9387 - val_loss: 0.3653 - val_acc: 0.8906\n",
      "Epoch 298/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1726 - acc: 0.9396 - val_loss: 0.3903 - val_acc: 0.8878\n",
      "Epoch 299/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1764 - acc: 0.9393 - val_loss: 0.3674 - val_acc: 0.8918\n",
      "Epoch 300/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1731 - acc: 0.9385 - val_loss: 0.3703 - val_acc: 0.8928\n",
      "Epoch 301/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1731 - acc: 0.9403 - val_loss: 0.3616 - val_acc: 0.8910\n",
      "Epoch 302/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1747 - acc: 0.9378 - val_loss: 0.3724 - val_acc: 0.8918\n",
      "Epoch 303/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1658 - acc: 0.9428 - val_loss: 0.3848 - val_acc: 0.8904\n",
      "Epoch 304/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1755 - acc: 0.9374 - val_loss: 0.3573 - val_acc: 0.8932\n",
      "Epoch 305/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1714 - acc: 0.9404 - val_loss: 0.3685 - val_acc: 0.8948\n",
      "Epoch 306/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1713 - acc: 0.9403 - val_loss: 0.3628 - val_acc: 0.8950\n",
      "Epoch 307/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1723 - acc: 0.9396 - val_loss: 0.3685 - val_acc: 0.8932\n",
      "Epoch 308/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1678 - acc: 0.9414 - val_loss: 0.3839 - val_acc: 0.8872\n",
      "Epoch 309/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1701 - acc: 0.9400 - val_loss: 0.3732 - val_acc: 0.8888\n",
      "Epoch 310/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1681 - acc: 0.9421 - val_loss: 0.3769 - val_acc: 0.8902\n",
      "Epoch 311/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1657 - acc: 0.9415 - val_loss: 0.3792 - val_acc: 0.8914\n",
      "Epoch 312/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1627 - acc: 0.9417 - val_loss: 0.3855 - val_acc: 0.8888\n",
      "Epoch 313/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1703 - acc: 0.9400 - val_loss: 0.3704 - val_acc: 0.8914\n",
      "Epoch 314/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1698 - acc: 0.9406 - val_loss: 0.3822 - val_acc: 0.8862\n",
      "Epoch 315/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1652 - acc: 0.9425 - val_loss: 0.3661 - val_acc: 0.8902\n",
      "Epoch 316/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1635 - acc: 0.9431 - val_loss: 0.3726 - val_acc: 0.8912\n",
      "Epoch 317/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1618 - acc: 0.9433 - val_loss: 0.3674 - val_acc: 0.8996\n",
      "Epoch 318/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1635 - acc: 0.9428 - val_loss: 0.3692 - val_acc: 0.8912\n",
      "Epoch 319/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1662 - acc: 0.9413 - val_loss: 0.3826 - val_acc: 0.8936\n",
      "Epoch 320/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1639 - acc: 0.9422 - val_loss: 0.3808 - val_acc: 0.8918\n",
      "Epoch 321/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1625 - acc: 0.9436 - val_loss: 0.3806 - val_acc: 0.8892\n",
      "Epoch 322/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1574 - acc: 0.9458 - val_loss: 0.3754 - val_acc: 0.8962\n",
      "Epoch 323/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1637 - acc: 0.9415 - val_loss: 0.3659 - val_acc: 0.8966\n",
      "Epoch 324/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1659 - acc: 0.9424 - val_loss: 0.3673 - val_acc: 0.8926\n",
      "Epoch 325/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1596 - acc: 0.9451 - val_loss: 0.3597 - val_acc: 0.8934\n",
      "Epoch 326/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1525 - acc: 0.9451 - val_loss: 0.3873 - val_acc: 0.8904\n",
      "Epoch 327/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1555 - acc: 0.9458 - val_loss: 0.3904 - val_acc: 0.8906\n",
      "Epoch 328/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1632 - acc: 0.9436 - val_loss: 0.3659 - val_acc: 0.8924\n",
      "Epoch 329/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1600 - acc: 0.9436 - val_loss: 0.3853 - val_acc: 0.8872\n",
      "Epoch 330/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1630 - acc: 0.9440 - val_loss: 0.3925 - val_acc: 0.8918\n",
      "Epoch 331/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1570 - acc: 0.9445 - val_loss: 0.3721 - val_acc: 0.8932\n",
      "Epoch 332/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1596 - acc: 0.9444 - val_loss: 0.3670 - val_acc: 0.8928\n",
      "Epoch 333/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1558 - acc: 0.9458 - val_loss: 0.3752 - val_acc: 0.8940\n",
      "Epoch 334/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1589 - acc: 0.9443 - val_loss: 0.3660 - val_acc: 0.8912\n",
      "Epoch 335/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1577 - acc: 0.9440 - val_loss: 0.3830 - val_acc: 0.8892\n",
      "Epoch 336/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1578 - acc: 0.9443 - val_loss: 0.3715 - val_acc: 0.8962\n",
      "Epoch 337/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1549 - acc: 0.9448 - val_loss: 0.3942 - val_acc: 0.8860\n",
      "Epoch 338/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1514 - acc: 0.9461 - val_loss: 0.3894 - val_acc: 0.8932\n",
      "Epoch 339/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1551 - acc: 0.9452 - val_loss: 0.3722 - val_acc: 0.8950\n",
      "Epoch 340/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1567 - acc: 0.9446 - val_loss: 0.3756 - val_acc: 0.8978\n",
      "Epoch 341/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1519 - acc: 0.9463 - val_loss: 0.3921 - val_acc: 0.8924\n",
      "Epoch 342/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1516 - acc: 0.9475 - val_loss: 0.3863 - val_acc: 0.8940\n",
      "Epoch 343/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1565 - acc: 0.9459 - val_loss: 0.3682 - val_acc: 0.8982\n",
      "Epoch 344/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1531 - acc: 0.9470 - val_loss: 0.3713 - val_acc: 0.8944\n",
      "Epoch 345/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1530 - acc: 0.9459 - val_loss: 0.3794 - val_acc: 0.8916\n",
      "Epoch 346/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1490 - acc: 0.9478 - val_loss: 0.3611 - val_acc: 0.8990\n",
      "Epoch 347/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1527 - acc: 0.9471 - val_loss: 0.3732 - val_acc: 0.8948\n",
      "Epoch 348/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1526 - acc: 0.9472 - val_loss: 0.3611 - val_acc: 0.8974\n",
      "Epoch 349/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1460 - acc: 0.9492 - val_loss: 0.3735 - val_acc: 0.8982\n",
      "Epoch 350/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1527 - acc: 0.9470 - val_loss: 0.3765 - val_acc: 0.8954\n",
      "Epoch 351/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1467 - acc: 0.9482 - val_loss: 0.3616 - val_acc: 0.8994\n",
      "Epoch 352/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1527 - acc: 0.9460 - val_loss: 0.3839 - val_acc: 0.8932\n",
      "Epoch 353/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1505 - acc: 0.9470 - val_loss: 0.3691 - val_acc: 0.8912\n",
      "Epoch 354/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1460 - acc: 0.9494 - val_loss: 0.3809 - val_acc: 0.8908\n",
      "Epoch 355/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1442 - acc: 0.9487 - val_loss: 0.3862 - val_acc: 0.8892\n",
      "Epoch 356/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1497 - acc: 0.9479 - val_loss: 0.3736 - val_acc: 0.8878\n",
      "Epoch 357/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1484 - acc: 0.9480 - val_loss: 0.3906 - val_acc: 0.8910\n",
      "Epoch 358/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1442 - acc: 0.9503 - val_loss: 0.4060 - val_acc: 0.8880\n",
      "Epoch 359/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1455 - acc: 0.9500 - val_loss: 0.3778 - val_acc: 0.8930\n",
      "Epoch 360/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1444 - acc: 0.9496 - val_loss: 0.3773 - val_acc: 0.8928\n",
      "Epoch 361/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1530 - acc: 0.9475 - val_loss: 0.3637 - val_acc: 0.8950\n",
      "Epoch 362/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1459 - acc: 0.9479 - val_loss: 0.3621 - val_acc: 0.8950\n",
      "Epoch 363/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1488 - acc: 0.9477 - val_loss: 0.3746 - val_acc: 0.8932\n",
      "Epoch 364/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1432 - acc: 0.9504 - val_loss: 0.3574 - val_acc: 0.8944\n",
      "Epoch 365/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1393 - acc: 0.9512 - val_loss: 0.3618 - val_acc: 0.8958\n",
      "Epoch 366/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1411 - acc: 0.9510 - val_loss: 0.3802 - val_acc: 0.8944\n",
      "Epoch 367/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1435 - acc: 0.9495 - val_loss: 0.4015 - val_acc: 0.8896\n",
      "Epoch 368/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1412 - acc: 0.9506 - val_loss: 0.3746 - val_acc: 0.8936\n",
      "Epoch 369/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1427 - acc: 0.9500 - val_loss: 0.3911 - val_acc: 0.8874\n",
      "Epoch 370/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1405 - acc: 0.9510 - val_loss: 0.3871 - val_acc: 0.8902\n",
      "Epoch 371/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1369 - acc: 0.9519 - val_loss: 0.3957 - val_acc: 0.8914\n",
      "Epoch 372/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1430 - acc: 0.9492 - val_loss: 0.3751 - val_acc: 0.8944\n",
      "Epoch 373/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1426 - acc: 0.9517 - val_loss: 0.3790 - val_acc: 0.8896\n",
      "Epoch 374/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1403 - acc: 0.9522 - val_loss: 0.3861 - val_acc: 0.8934\n",
      "Epoch 375/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1449 - acc: 0.9485 - val_loss: 0.3595 - val_acc: 0.8944\n",
      "Epoch 376/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1383 - acc: 0.9520 - val_loss: 0.3707 - val_acc: 0.8956\n",
      "Epoch 377/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1375 - acc: 0.9523 - val_loss: 0.3956 - val_acc: 0.8924\n",
      "Epoch 378/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1446 - acc: 0.9489 - val_loss: 0.3739 - val_acc: 0.8946\n",
      "Epoch 379/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1379 - acc: 0.9521 - val_loss: 0.3489 - val_acc: 0.8954\n",
      "Epoch 380/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1393 - acc: 0.9516 - val_loss: 0.3792 - val_acc: 0.8962\n",
      "Epoch 381/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1372 - acc: 0.9532 - val_loss: 0.3983 - val_acc: 0.8930\n",
      "Epoch 382/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1390 - acc: 0.9512 - val_loss: 0.4203 - val_acc: 0.8828\n",
      "Epoch 383/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1384 - acc: 0.9510 - val_loss: 0.3824 - val_acc: 0.8906\n",
      "Epoch 384/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1375 - acc: 0.9517 - val_loss: 0.3805 - val_acc: 0.8934\n",
      "Epoch 385/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1387 - acc: 0.9520 - val_loss: 0.3843 - val_acc: 0.8932\n",
      "Epoch 386/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1361 - acc: 0.9533 - val_loss: 0.3915 - val_acc: 0.8902\n",
      "Epoch 387/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1374 - acc: 0.9527 - val_loss: 0.3578 - val_acc: 0.8970\n",
      "Epoch 388/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1396 - acc: 0.9517 - val_loss: 0.3723 - val_acc: 0.8926\n",
      "Epoch 389/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1363 - acc: 0.9527 - val_loss: 0.3863 - val_acc: 0.8918\n",
      "Epoch 390/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1351 - acc: 0.9532 - val_loss: 0.4091 - val_acc: 0.8910\n",
      "Epoch 391/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1392 - acc: 0.9503 - val_loss: 0.3747 - val_acc: 0.8964\n",
      "Epoch 392/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1362 - acc: 0.9526 - val_loss: 0.3934 - val_acc: 0.8962\n",
      "Epoch 393/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1382 - acc: 0.9516 - val_loss: 0.3639 - val_acc: 0.8930\n",
      "Epoch 394/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1306 - acc: 0.9535 - val_loss: 0.3868 - val_acc: 0.8922\n",
      "Epoch 395/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1337 - acc: 0.9534 - val_loss: 0.3949 - val_acc: 0.8940\n",
      "Epoch 396/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1286 - acc: 0.9546 - val_loss: 0.3844 - val_acc: 0.8978\n",
      "Epoch 397/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1293 - acc: 0.9559 - val_loss: 0.3716 - val_acc: 0.8960\n",
      "Epoch 398/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1325 - acc: 0.9543 - val_loss: 0.4029 - val_acc: 0.8948\n",
      "Epoch 399/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1329 - acc: 0.9538 - val_loss: 0.3664 - val_acc: 0.8946\n",
      "Epoch 400/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1337 - acc: 0.9529 - val_loss: 0.3629 - val_acc: 0.8934\n",
      "Epoch 401/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1307 - acc: 0.9540 - val_loss: 0.3761 - val_acc: 0.8926\n",
      "Epoch 402/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1298 - acc: 0.9552 - val_loss: 0.3846 - val_acc: 0.8938\n",
      "Epoch 403/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1292 - acc: 0.9548 - val_loss: 0.3927 - val_acc: 0.8934\n",
      "Epoch 404/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1317 - acc: 0.9540 - val_loss: 0.3837 - val_acc: 0.8908\n",
      "Epoch 405/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1303 - acc: 0.9549 - val_loss: 0.4046 - val_acc: 0.8942\n",
      "Epoch 406/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1293 - acc: 0.9543 - val_loss: 0.4018 - val_acc: 0.8940\n",
      "Epoch 407/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1306 - acc: 0.9559 - val_loss: 0.4087 - val_acc: 0.8936\n",
      "Epoch 408/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1270 - acc: 0.9559 - val_loss: 0.3862 - val_acc: 0.8952\n",
      "Epoch 409/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1327 - acc: 0.9556 - val_loss: 0.3956 - val_acc: 0.8894\n",
      "Epoch 410/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1319 - acc: 0.9541 - val_loss: 0.3878 - val_acc: 0.8918\n",
      "Epoch 411/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1275 - acc: 0.9557 - val_loss: 0.3840 - val_acc: 0.8944\n",
      "Epoch 412/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1306 - acc: 0.9548 - val_loss: 0.3831 - val_acc: 0.8950\n",
      "Epoch 413/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1247 - acc: 0.9572 - val_loss: 0.3897 - val_acc: 0.8910\n",
      "Epoch 414/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1291 - acc: 0.9548 - val_loss: 0.3740 - val_acc: 0.8972\n",
      "Epoch 415/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1255 - acc: 0.9560 - val_loss: 0.3924 - val_acc: 0.8906\n",
      "Epoch 416/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1232 - acc: 0.9568 - val_loss: 0.3760 - val_acc: 0.8996\n",
      "Epoch 417/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1275 - acc: 0.9551 - val_loss: 0.4021 - val_acc: 0.8936\n",
      "Epoch 418/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1235 - acc: 0.9564 - val_loss: 0.4006 - val_acc: 0.8956\n",
      "Epoch 419/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1261 - acc: 0.9556 - val_loss: 0.3900 - val_acc: 0.8896\n",
      "Epoch 420/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1279 - acc: 0.9558 - val_loss: 0.3885 - val_acc: 0.8938\n",
      "Epoch 421/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1232 - acc: 0.9561 - val_loss: 0.4003 - val_acc: 0.8968\n",
      "Epoch 422/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1270 - acc: 0.9556 - val_loss: 0.4042 - val_acc: 0.8898\n",
      "Epoch 423/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1268 - acc: 0.9562 - val_loss: 0.3988 - val_acc: 0.8982\n",
      "Epoch 424/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1222 - acc: 0.9575 - val_loss: 0.3890 - val_acc: 0.8910\n",
      "Epoch 425/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1234 - acc: 0.9568 - val_loss: 0.3811 - val_acc: 0.8958\n",
      "Epoch 426/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1240 - acc: 0.9579 - val_loss: 0.3934 - val_acc: 0.8880\n",
      "Epoch 427/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1304 - acc: 0.9551 - val_loss: 0.3889 - val_acc: 0.8922\n",
      "Epoch 428/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1195 - acc: 0.9586 - val_loss: 0.4037 - val_acc: 0.8906\n",
      "Epoch 429/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1249 - acc: 0.9562 - val_loss: 0.3909 - val_acc: 0.8952\n",
      "Epoch 430/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1237 - acc: 0.9565 - val_loss: 0.4089 - val_acc: 0.8860\n",
      "Epoch 431/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1236 - acc: 0.9577 - val_loss: 0.4113 - val_acc: 0.8948\n",
      "Epoch 432/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1215 - acc: 0.9584 - val_loss: 0.3844 - val_acc: 0.8966\n",
      "Epoch 433/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1250 - acc: 0.9577 - val_loss: 0.3988 - val_acc: 0.8960\n",
      "Epoch 434/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1250 - acc: 0.9577 - val_loss: 0.4051 - val_acc: 0.8944\n",
      "Epoch 435/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1193 - acc: 0.9577 - val_loss: 0.4081 - val_acc: 0.8914\n",
      "Epoch 436/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1223 - acc: 0.9579 - val_loss: 0.3847 - val_acc: 0.8940\n",
      "Epoch 437/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1198 - acc: 0.9587 - val_loss: 0.3941 - val_acc: 0.8910\n",
      "Epoch 438/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1222 - acc: 0.9579 - val_loss: 0.3975 - val_acc: 0.8936\n",
      "Epoch 439/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1232 - acc: 0.9578 - val_loss: 0.3854 - val_acc: 0.8962\n",
      "Epoch 440/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1231 - acc: 0.9565 - val_loss: 0.3863 - val_acc: 0.8906\n",
      "Epoch 441/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1179 - acc: 0.9602 - val_loss: 0.3822 - val_acc: 0.8920\n",
      "Epoch 442/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1170 - acc: 0.9594 - val_loss: 0.4101 - val_acc: 0.8920\n",
      "Epoch 443/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1203 - acc: 0.9587 - val_loss: 0.4008 - val_acc: 0.8914\n",
      "Epoch 444/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1214 - acc: 0.9572 - val_loss: 0.4031 - val_acc: 0.8878\n",
      "Epoch 445/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1220 - acc: 0.9581 - val_loss: 0.3891 - val_acc: 0.8920\n",
      "Epoch 446/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1210 - acc: 0.9589 - val_loss: 0.4028 - val_acc: 0.8948\n",
      "Epoch 447/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1144 - acc: 0.9597 - val_loss: 0.3704 - val_acc: 0.8972\n",
      "Epoch 448/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1158 - acc: 0.9587 - val_loss: 0.3907 - val_acc: 0.8942\n",
      "Epoch 449/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1199 - acc: 0.9589 - val_loss: 0.3861 - val_acc: 0.8992\n",
      "Epoch 450/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1169 - acc: 0.9596 - val_loss: 0.4063 - val_acc: 0.8944\n",
      "Epoch 451/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1167 - acc: 0.9606 - val_loss: 0.3899 - val_acc: 0.8972\n",
      "Epoch 452/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1163 - acc: 0.9591 - val_loss: 0.4020 - val_acc: 0.8924\n",
      "Epoch 453/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1169 - acc: 0.9605 - val_loss: 0.3809 - val_acc: 0.8970\n",
      "Epoch 454/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1216 - acc: 0.9590 - val_loss: 0.3736 - val_acc: 0.8958\n",
      "Epoch 455/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1172 - acc: 0.9592 - val_loss: 0.3822 - val_acc: 0.8946\n",
      "Epoch 456/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1160 - acc: 0.9602 - val_loss: 0.3911 - val_acc: 0.8960\n",
      "Epoch 457/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1224 - acc: 0.9587 - val_loss: 0.3847 - val_acc: 0.8940\n",
      "Epoch 458/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1156 - acc: 0.9593 - val_loss: 0.4040 - val_acc: 0.8884\n",
      "Epoch 459/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1166 - acc: 0.9605 - val_loss: 0.3932 - val_acc: 0.8936\n",
      "Epoch 460/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1186 - acc: 0.9583 - val_loss: 0.3896 - val_acc: 0.8938\n",
      "Epoch 461/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1147 - acc: 0.9599 - val_loss: 0.3921 - val_acc: 0.8934\n",
      "Epoch 462/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1158 - acc: 0.9611 - val_loss: 0.3963 - val_acc: 0.8904\n",
      "Epoch 463/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1127 - acc: 0.9616 - val_loss: 0.3885 - val_acc: 0.8964\n",
      "Epoch 464/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1142 - acc: 0.9609 - val_loss: 0.3966 - val_acc: 0.8924\n",
      "Epoch 465/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1133 - acc: 0.9603 - val_loss: 0.3942 - val_acc: 0.8968\n",
      "Epoch 466/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1193 - acc: 0.9592 - val_loss: 0.3900 - val_acc: 0.8932\n",
      "Epoch 467/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1134 - acc: 0.9614 - val_loss: 0.4135 - val_acc: 0.8944\n",
      "Epoch 468/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1098 - acc: 0.9621 - val_loss: 0.4094 - val_acc: 0.8924\n",
      "Epoch 469/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1137 - acc: 0.9604 - val_loss: 0.4012 - val_acc: 0.8980\n",
      "Epoch 470/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1132 - acc: 0.9621 - val_loss: 0.3999 - val_acc: 0.8956\n",
      "Epoch 471/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1160 - acc: 0.9604 - val_loss: 0.3697 - val_acc: 0.8988\n",
      "Epoch 472/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1102 - acc: 0.9619 - val_loss: 0.4055 - val_acc: 0.8922\n",
      "Epoch 473/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1090 - acc: 0.9628 - val_loss: 0.4237 - val_acc: 0.8908\n",
      "Epoch 474/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1139 - acc: 0.9598 - val_loss: 0.3902 - val_acc: 0.8944\n",
      "Epoch 475/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1092 - acc: 0.9613 - val_loss: 0.4120 - val_acc: 0.8960\n",
      "Epoch 476/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1126 - acc: 0.9619 - val_loss: 0.4002 - val_acc: 0.8940\n",
      "Epoch 477/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1124 - acc: 0.9612 - val_loss: 0.4397 - val_acc: 0.8842\n",
      "Epoch 478/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1129 - acc: 0.9608 - val_loss: 0.3776 - val_acc: 0.8990\n",
      "Epoch 479/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1100 - acc: 0.9624 - val_loss: 0.3877 - val_acc: 0.8936\n",
      "Epoch 480/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1084 - acc: 0.9629 - val_loss: 0.3908 - val_acc: 0.8952\n",
      "Epoch 481/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1113 - acc: 0.9618 - val_loss: 0.4053 - val_acc: 0.8936\n",
      "Epoch 482/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1118 - acc: 0.9625 - val_loss: 0.3951 - val_acc: 0.8938\n",
      "Epoch 483/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1112 - acc: 0.9611 - val_loss: 0.4099 - val_acc: 0.8892\n",
      "Epoch 484/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1109 - acc: 0.9617 - val_loss: 0.3889 - val_acc: 0.8936\n",
      "Epoch 485/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1098 - acc: 0.9628 - val_loss: 0.4038 - val_acc: 0.8942\n",
      "Epoch 486/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1108 - acc: 0.9614 - val_loss: 0.3955 - val_acc: 0.8986\n",
      "Epoch 487/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1120 - acc: 0.9612 - val_loss: 0.3749 - val_acc: 0.8972\n",
      "Epoch 488/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1114 - acc: 0.9617 - val_loss: 0.3898 - val_acc: 0.8942\n",
      "Epoch 489/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1078 - acc: 0.9619 - val_loss: 0.3988 - val_acc: 0.8938\n",
      "Epoch 490/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1073 - acc: 0.9640 - val_loss: 0.4182 - val_acc: 0.8920\n",
      "Epoch 491/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1104 - acc: 0.9625 - val_loss: 0.3975 - val_acc: 0.8936\n",
      "Epoch 492/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1079 - acc: 0.9617 - val_loss: 0.3941 - val_acc: 0.8940\n",
      "Epoch 493/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1084 - acc: 0.9638 - val_loss: 0.4017 - val_acc: 0.8910\n",
      "Epoch 494/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1091 - acc: 0.9623 - val_loss: 0.3906 - val_acc: 0.8930\n",
      "Epoch 495/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1142 - acc: 0.9619 - val_loss: 0.3721 - val_acc: 0.8988\n",
      "Epoch 496/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1098 - acc: 0.9617 - val_loss: 0.4099 - val_acc: 0.8906\n",
      "Epoch 497/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1099 - acc: 0.9626 - val_loss: 0.4199 - val_acc: 0.8910\n",
      "Epoch 498/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1126 - acc: 0.9619 - val_loss: 0.4141 - val_acc: 0.8934\n",
      "Epoch 499/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1094 - acc: 0.9622 - val_loss: 0.4071 - val_acc: 0.8940\n",
      "Epoch 500/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1068 - acc: 0.9633 - val_loss: 0.4107 - val_acc: 0.8918\n",
      "Epoch 501/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1037 - acc: 0.9640 - val_loss: 0.4297 - val_acc: 0.8912\n",
      "Epoch 502/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1098 - acc: 0.9622 - val_loss: 0.4171 - val_acc: 0.8942\n",
      "Epoch 503/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1060 - acc: 0.9638 - val_loss: 0.4012 - val_acc: 0.8950\n",
      "Epoch 504/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1104 - acc: 0.9622 - val_loss: 0.4101 - val_acc: 0.8930\n",
      "Epoch 505/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1079 - acc: 0.9635 - val_loss: 0.3940 - val_acc: 0.8982\n",
      "Epoch 506/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1063 - acc: 0.9637 - val_loss: 0.4010 - val_acc: 0.8988\n",
      "Epoch 507/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1076 - acc: 0.9630 - val_loss: 0.4064 - val_acc: 0.8894\n",
      "Epoch 508/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1060 - acc: 0.9637 - val_loss: 0.4222 - val_acc: 0.8958\n",
      "Epoch 509/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1078 - acc: 0.9643 - val_loss: 0.4145 - val_acc: 0.9004\n",
      "Epoch 510/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1067 - acc: 0.9633 - val_loss: 0.4047 - val_acc: 0.8934\n",
      "Epoch 511/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1050 - acc: 0.9632 - val_loss: 0.4037 - val_acc: 0.8916\n",
      "Epoch 512/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1060 - acc: 0.9639 - val_loss: 0.3894 - val_acc: 0.8896\n",
      "Epoch 513/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1070 - acc: 0.9633 - val_loss: 0.3895 - val_acc: 0.8946\n",
      "Epoch 514/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1046 - acc: 0.9639 - val_loss: 0.3947 - val_acc: 0.8974\n",
      "Epoch 515/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1072 - acc: 0.9631 - val_loss: 0.3819 - val_acc: 0.8962\n",
      "Epoch 516/1600\n",
      "351/351 [==============================] - 11s - loss: 0.0998 - acc: 0.9657 - val_loss: 0.4111 - val_acc: 0.8924\n",
      "Epoch 517/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1044 - acc: 0.9642 - val_loss: 0.4124 - val_acc: 0.8928\n",
      "Epoch 518/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1040 - acc: 0.9643 - val_loss: 0.4205 - val_acc: 0.8902\n",
      "Epoch 519/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1045 - acc: 0.9637 - val_loss: 0.3852 - val_acc: 0.8982\n",
      "Epoch 520/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1064 - acc: 0.9643 - val_loss: 0.4217 - val_acc: 0.8898\n",
      "Epoch 521/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1010 - acc: 0.9656 - val_loss: 0.4249 - val_acc: 0.8894\n",
      "Epoch 522/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1059 - acc: 0.9633 - val_loss: 0.3986 - val_acc: 0.8968\n",
      "Epoch 523/1600\n",
      "351/351 [==============================] - 11s - loss: 0.0995 - acc: 0.9657 - val_loss: 0.4108 - val_acc: 0.8922\n",
      "Epoch 524/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1031 - acc: 0.9642 - val_loss: 0.4352 - val_acc: 0.8900\n",
      "Epoch 525/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1064 - acc: 0.9643 - val_loss: 0.4102 - val_acc: 0.8908\n",
      "Epoch 526/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1033 - acc: 0.9648 - val_loss: 0.4010 - val_acc: 0.8958\n",
      "Epoch 527/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1039 - acc: 0.9633 - val_loss: 0.4146 - val_acc: 0.8902\n",
      "Epoch 528/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1053 - acc: 0.9644 - val_loss: 0.3979 - val_acc: 0.8968\n",
      "Epoch 529/1600\n",
      "351/351 [==============================] - 11s - loss: 0.1080 - acc: 0.9627 - val_loss: 0.3890 - val_acc: 0.8922\n",
      "Epoch 530/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1069 - acc: 0.9631 - val_loss: 0.3770 - val_acc: 0.8946\n",
      "Epoch 531/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1026 - acc: 0.9641 - val_loss: 0.3903 - val_acc: 0.8954\n",
      "Epoch 532/1600\n",
      "351/351 [==============================] - 12s - loss: 0.1007 - acc: 0.9659 - val_loss: 0.3962 - val_acc: 0.8978\n",
      "Epoch 533/1600\n",
      " 21/351 [>.............................] - ETA: 11s - loss: 0.1144 - acc: 0.9568"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-0ffcd16c74ce>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    112\u001b[0m                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mx_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m//\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    113\u001b[0m                         \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 114\u001b[1;33m                         validation_data=(x_test, y_test), callbacks=[tbCallBack])\n\u001b[0m",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[0;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 87\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\keras\\models.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, initial_epoch)\u001b[0m\n\u001b[0;32m   1115\u001b[0m                                         \u001b[0mworkers\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1116\u001b[0m                                         \u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1117\u001b[1;33m                                         initial_epoch=initial_epoch)\n\u001b[0m\u001b[0;32m   1118\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1119\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[0;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 87\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, initial_epoch)\u001b[0m\n\u001b[0;32m   1838\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[0;32m   1839\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1840\u001b[1;33m                                                class_weight=class_weight)\n\u001b[0m\u001b[0;32m   1841\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1842\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[1;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[0;32m   1563\u001b[0m             \u001b[0mins\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1564\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1565\u001b[1;33m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1566\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1567\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2266\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m   2267\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2268\u001b[1;33m                               **self.session_kwargs)\n\u001b[0m\u001b[0;32m   2269\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2270\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    787\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 789\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    790\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    995\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m--> 997\u001b[1;33m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[0;32m    998\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1130\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[1;32m-> 1132\u001b[1;33m                            target_list, options, run_metadata)\n\u001b[0m\u001b[0;32m   1133\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1137\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1138\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1139\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1140\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\anaconda2\\envs\\tfgpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[0;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1121\u001b[1;33m                                  status, run_metadata)\n\u001b[0m\u001b[0;32m   1122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1123\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D, GlobalMaxPooling2D\n",
    "from lsuv_init import LSUVinit\n",
    "import pickle\n",
    "\n",
    "batch_size = 128 \n",
    "num_classes = 10\n",
    "epochs = 1600\n",
    "data_augmentation = True\n",
    "\n",
    "# The data, shuffled and split between train and test sets:\n",
    "x_train, y_train, x_test, y_test = pickle.load(open('CIFAR10 Preprocessed', mode='rb'))\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(48, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(48, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(80, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(80, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(80, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(80, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(80, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(128, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(128, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(128, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(128, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(128, (3, 3), padding='same', input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(GlobalMaxPooling2D())\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Dense(500))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "# initiate RMSprop optimizer\n",
    "opt = keras.optimizers.Adam(lr=0.0001)\n",
    "\n",
    "# Let's train the model using RMSprop\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "model = LSUVinit(model,x_train[:batch_size,:,:,:]) \n",
    "tbCallBack = keras.callbacks.TensorBoard(log_dir='./Graph2', histogram_freq=0, write_graph=True, write_images=True)\n",
    "\n",
    "if not data_augmentation:\n",
    "    print('Not using data augmentation.')\n",
    "    model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True, callbacks=[tbCallBack])\n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        width_shift_range=0.2,  # randomly shift images horizontally (fraction of total width)\n",
    "        height_shift_range=0.2,  # randomly shift images vertically (fraction of total height)\n",
    "        horizontal_flip=True,  # randomly flip images\n",
    "        vertical_flip=False)  # randomly flip images\n",
    "\n",
    "\n",
    "    # Compute quantities required for feature-wise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train, y_train,\n",
    "                                     batch_size=batch_size),\n",
    "                        steps_per_epoch=x_train.shape[0] // batch_size,\n",
    "                        epochs=epochs,\n",
    "                        validation_data=(x_test, y_test), callbacks=[tbCallBack])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
